
var request = require('request');
var async = require('async');
var driver = require('./db');
var Iconv = require('iconv-lite');
var url_node = require('url');
var utils = require('./utils');
/*
var page = {
    curPage: 1,
    totalPage: 259,
};
var url = "http://cts.dajie.com/cts/job/list?ajax=1&corpId=9104102800000000751&pageNo=*PageReplace*&orgCode=&type=&workPlace=&_CSRFToken=";
*/


// 汲取
/**
 * 
 * @param {*} option {url:url,type:type,headers:headers,timeout:timeout}||url
 * @param {*} callback 
 */
function spider(option, callback) {

    // 存入的是url
    if(typeof(option)=='string'){
        var url = option;
        option = {
            url:url
        };
        url = null;
    }

    option.headers = option.headers ? option.headers : {};
    option.timeout = option.timeout ? option.timeout : 19000;

    console.log("spider url:"+option.url);

    request({ url: option.url, headers: option.headers, encoding: null, timeout: option.timeout }, function (err, res, body) {

        if (err) {

            console.log(err);
            callback(err, 'error' + option.url);
            return;
        }

        callback(null, body,option.url);

    });

}



// 翻页
/**
 * 
 * @param {*} url 
 * @param {*} page {curPage:1,totalPage:89}
 * @param {*} callback 
 */
function pageNext() {

    var url = arguments[0];
    var page = arguments[1];
    var callback = arguments[2];
    var headers = arguments[3] ||{};
    var successCall = arguments[4] || function(){};

    console.log("total page:" + page.totalPage);



    // 翻页
    if (page.curPage < page.totalPage) {

        var _url = url.replace('*PageReplace*', page.curPage);

        spider({ url: _url,headers:headers }, callback);

        // 递归
        setTimeout(() => {
            page.curPage++;
            pageNext(url, page, callback);
        }, 5000);
    } else {
        
        var _url = url.replace('*PageReplace*', page.curPage);

        spider({ url: _url }, callback);
       

        console.log('-----------spider all urls-----------------');
    }
}


module.exports = {
    spider: spider,
    pageNext: pageNext
};